#!/usr/bin/python
import sys
from itertools import groupby, ifilter

def get_text(file):
    """ Read and split file
    """
    try:
        f = open(file, 'r')
        text = f.read().split()
        f.close()
    except IOError:
        print "IO ERROR:", file, "could not be read."
        sys.exit(1)
    return text

def n_gram(text, n):
    """ Yield n-grams from word list
    """
    #for i in xrange(len(text)-(n-1)):
    #    yield ' '.join(text[i:i+n])
   
    """  create n-grams from word list
    """
    nGrams = []
    for i in xrange(len(text)-(n-1)):
        nGrams += [' '.join(text[i:i+n])]
    return nGrams

def freq_list(nGrams):
    """ calc frequency of occurence of ngrams
    """
    # creates list of tuples in the form (ngram, frequency)
    freqList = [(k, len(list(g))) for k,g in groupby(sorted(nGrams))]
    # sort tuples according to occurences
    freqList = sorted(freqList, key=lambda strFreq: strFreq[1], reverse=True)
    return freqList

def get_args(argv):
    """ transforms the arguments
    """
    if len(argv) > 1:
        file = argv[1]
    else:
        file = "austen.txt"
    return file

def print_info():
    """ Returns general information
    """
    print "Taalmodellen opdracht II: Extracting ngrams"
    print "authors: Chiel Kooijman 5743028, Niklas Semmler 6191177"

def main(argv=sys.argv):
    """ all the work is done here 
    """
    file = get_args(argv)
    text = get_text(file)
    
    oneGrams = n_gram(text, 1)
    oneFreqList = freq_list(oneGrams)

    twoGrams = n_gram(text, 2)
    twoFreqList = freq_list(twoGrams)

    threeGrams = n_gram(text, 3)
    threeFreqList = freq_list(threeGrams)
   
    print_info()
    print "Question 1"
    print "n = 1 ", oneFreqList[0:10]
    print "n = 2 ", twoFreqList[0:10]
    print "n = 3 ", threeFreqList[0:10]
    
    print "Question 2"
    print "n = 1 ", sum(freq for (word,freq) in oneFreqList)
    print "n = 2 ", sum(freq for (word,freq) in twoFreqList)
    print "n = 3 ", sum(freq for (word,freq) in threeFreqList)
    
    print "Question 3"
    print "n = 1 & m = 1 ", sum(ifilter(lambda k: k==1, (freq for (word,freq) in oneFreqList)))
    print "n = 1 & m = 2 ", sum(ifilter(lambda k: k==2, (freq for (word,freq) in oneFreqList)))

if __name__ == "__main__":
    sys.exit(main())
